import os
os.environ["OMP_NUM_THREADS"] = "1"
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["NUMEXPR_NUM_THREADS"] = "1"
os.environ["ARROW_NUM_THREADS"] = "1"
import pandas as pd
import glob
import time
from multiprocessing import Pool, cpu_count
import numpy as np
from tqdm import tqdm
from typing import List, Dict, Set, Tuple, Optional
from dataclasses import dataclass
import logging

# 配置日志
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

@dataclass
class Config:
    """配置类，集中管理所有参数"""
    num_processes: int = min(8, cpu_count())
    input_dir: str = "30G_data"
    output_dir: str = "processed_data_terminus"  # 原目录名: 30G_processed_data_new_terminus
    chunk_size: int = 100000  # 每次读取的行数
    max_rows: Optional[int] = None  # 最大处理行数，None表示处理所有数据

# ... [其余代码保持不变] ...

if __name__ == "__main__":
    logger.info("启动数据处理程序 (data_processor_terminus.py)")
    main() 